from util import *
import pandas as pd


def get_most_frequent_ms_kind(count):
    root_dir=get_root_dir()
    df_all=pd.read_csv(root_dir+"/data_process/datas/statistic_dm_request_number.csv")
    df_all.sort_values(by="traceid_count",ascending=False, inplace=True)
    df_all=df_all[~(df_all['dm'].isin(['UNAVAILABLE','UNKNOWN']))]
    most_frequent_ms_kind=df_all["dm"].head(count).tolist()
    return most_frequent_ms_kind
    
def get_mcr_pd(dir_path):
    global most_frequent_ms_kind
    # print(most_frequent_ms_kind)
    df_all=pd.DataFrame()
    tar_file_name=list_tar_files(dir_path)
    for file in tar_file_name:
        print(file)
        if test_flage==True:
        # df_temp=read_csv_from_tar_gz(dir_path+"/"+file)
            df_temp = pd.read_csv(os.path.join(dataset_dir,dir)+"/MSRTMCR_20.tar.gz.csv")
        else:
            df_temp=read_csv_from_tar_gz(dir_path+"/"+file)
        # print(df_temp['msname'].isin(most_frequent_ms_kind))
        #timestamp,msname,msinstanceid,providerrpc_mcr,consumerrpc_mcr,writemc_mcr,readmc_mcr,writedb_mcr,readdb_mcr,consumermq_mcr,providermq_mcr,http_mcr
        # df_temp=df_temp[df_temp['msname'].isin(most_frequent_ms_kind)]
        
        df_temp=df_temp[df_temp.apply(lambda x: str(x['msinstanceid']).startswith(str(x['msname'])), axis=1)]
        df_temp=df_temp[['timestamp','msname','msinstanceid','providerrpc_mcr','consumerrpc_mcr','writemc_mcr',\
            'readmc_mcr','writedb_mcr','readdb_mcr','consumermq_mcr','providermq_mcr','http_mcr']]
        df_temp['sum_mcr']=df_temp['providerrpc_mcr']+df_temp['consumerrpc_mcr']+df_temp['writemc_mcr']+df_temp['readmc_mcr']+\
            df_temp['writedb_mcr']+df_temp['readdb_mcr']+df_temp['consumermq_mcr']+df_temp['providermq_mcr']+df_temp['http_mcr']
        if df_all.empty:
            df_all = df_temp
        else:
            df_all = pd.concat([df_all, df_temp])
    # print(df_all.sort_values('sum_mcr', ascending=False))
    most_frequent_ms_kind = df_all.sort_values('sum_mcr', ascending=False)['msname'].drop_duplicates().head(count).tolist()
    print(f"get_mcr_pd {most_frequent_ms_kind}")
    df_all=df_all[df_all['msname'].isin(most_frequent_ms_kind)]
    
    return df_all

def get_utilization_pd(dir_path):
    global most_frequent_ms_kind
    print(f"get_utilization_pd {most_frequent_ms_kind}")
    df_all=pd.DataFrame()
    tar_file_name=list_tar_files(dir_path)
    for file in tar_file_name:
        print(file)
        if test_flage==True:
            df_temp = pd.read_csv(os.path.join(dataset_dir,dir)+"/MSMetrics_2.tar.gz.csv")
        else:
            df_temp=read_csv_from_tar_gz(dir_path+"/"+file)
        df_temp=df_temp[df_temp['msname'].isin(most_frequent_ms_kind)]
        df_temp=df_temp[df_temp.apply(lambda x: str(x['msinstanceid']).startswith(str(x['msname'])), axis=1)]
        df_temp=df_temp[['timestamp','msname','msinstanceid','cpu_utilization','memory_utilization']]
        df_temp['sum_utilization']=df_temp['cpu_utilization']+df_temp['memory_utilization']
        if df_all.empty:
            df_all = df_temp
        else:
            df_all = pd.concat([df_all, df_temp])
    
    return df_all




root_dir=get_root_dir()
dataset_dir=os.path.dirname(root_dir)+"/MS_data/data"
test_flage=False
# most_frequent_ms_kind=get_most_frequent_ms_kind(10)
# most_frequent_ms_kind.append("MS_72474")
# print(most_frequent_ms_kind)
count=5
most_frequent_ms_kind=[]
dir="MSRTMCR"
all_mcr_pd=get_mcr_pd(os.path.join(dataset_dir,dir))
dir="MSMetrics"
all_utilization_pd=get_utilization_pd(os.path.join(dataset_dir,dir))
merged_df = all_utilization_pd.merge(all_mcr_pd, on=['msinstanceid', 'timestamp','msname'], how='inner')
merged_df.to_csv(root_dir+"/data_process/datas/merged_data.csv",index=False)
print(merged_df)








